{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Preprocess.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZSK6_URGzKHd"
      },
      "outputs": [],
      "source": [
        "cd drive/MyDrive/"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "FILE1 = './CCKS_2019_Task1/subtask1_training_part1.txt'\n",
        "FILE2 = './CCKS_2019_Task1/subtask1_training_part2.txt'\n",
        "FILE3 = './CCKS_2019_Task1/subtask1_test_set_with_answer.json'\n",
        "\n",
        "PATH1 = './CCKS_2019_Task1/data/data1-'\n",
        "PATH2 = './CCKS_2019_Task1/data/data2-'\n",
        "PATH3 = './CCKS_2019_Task1/data_test/data-test-'\n",
        "\n",
        "def Process_File(FILENAME, PATH, enc):\n",
        "  with open(FILENAME, 'r', encoding=enc) as f:\n",
        "      i = 0\n",
        "      while True:\n",
        "        txt = f.readline()\n",
        "        if not txt: break         # end loop\n",
        "        i+=1\n",
        "        j = json.loads(txt)\n",
        "        orig = j['originalText']  # original text\n",
        "        entities = j['entities']  # entity part\n",
        "        pathO = PATH + str(i) + '-original.txt'\n",
        "        pathE = PATH + str(i) + '.txt'\n",
        "\n",
        "        with open(pathO, 'w', encoding='utf-8') as o1:  # write the original text\n",
        "            o1.write(orig)\n",
        "            o1.flush\n",
        "\n",
        "        with open(pathE, 'w', encoding='utf-8') as o2:  # wirte entity file\n",
        "            for e in entities:\n",
        "              start = e['start_pos']   # extract start position\n",
        "              end = e['end_pos']       # extract end position\n",
        "              name = orig[start:end]   # entity content\n",
        "              ty = e['label_type']     # entity label type\n",
        "              label = '{0}\\t{1}\\t{2}\\t{3}\\n'.format(name, start, end, ty)\n",
        "              o2.write(label)           \n",
        "              o2.flush"
      ],
      "metadata": {
        "id": "2XGlZ1RwzQdj"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Process_File(FILE1, PATH1, 'utf-8-sig')\n",
        "Process_File(FILE2, PATH2, 'utf-8-sig')\n",
        "Process_File(FILE3, PATH3, 'utf-8')"
      ],
      "metadata": {
        "id": "AIChvi6azqwV"
      },
      "execution_count": 3,
      "outputs": []
    }
  ]
}